# 示例：使用pyspark.mllib.recommendation做推荐案例-实现流程
# 地址：https://blog.csdn.net/eylier/article/details/105302693

# 1、初始化PySpark

# Initializing PySpark
from pyspark import SparkContext, SparkConf

conf = SparkConf().setMaster('local[4]').setAppName('movies_app')
sc = SparkContext(conf=conf)
import matplotlib.pyplot as plt
import numpy as np

# 2、加载数据，并做数据统计、绘制分布直方图、查看数据整体分布情况

# # 加载数据
# user_data = sc.textFile("/Users/onlyone/python/ml100k/u.user")
# user_data.first()
# # 统计数据
# user_fields = user_data.map(lambda line: line.split("|"))
# num_users = user_fields.map(lambda fields: fields[0]).count()
# num_genders = user_fields.map(lambda fields: fields[2]).distinct().count()
# num_occupations = user_fields.map(lambda fields: fields[3]).distinct().count()
# num_zipcodes = user_fields.map(lambda fields: fields[4]).distinct().count()
# print(
#     "Users: %d, genders: %d, occupations: %d, ZIP codes: %d" % (num_users, num_genders, num_occupations, num_zipcodes))
#
# # 建立分布直方图做统计
# ages = user_fields.map(lambda x: int(x[1])).collect()
# fig = plt.gcf()
# fig.set_size_inches(8, 6)
# plt.hist(ages, bins=20, color='lightblue', density=True, stacked=True)
# plt.title('histogram of ages')
# plt.show()

# 4、训练、评价模型

# 使用推荐模型接口
from pyspark.mllib.recommendation import ALS, Rating

rawData = sc.textFile("/Users/onlyone/python/ml100k/u.data")
rawRatings = rawData.map(lambda line: line.split("\t")[:3])
# 构造user-item-rating 数据
ratings = rawRatings.map(lambda line: Rating(user=int(line[0]), product=int(line[1]), rating=float(line[2])))

# 模型训练
model = ALS.train(ratings, rank=50, iterations=10, lambda_=0.01)

# 在训练集上评价模型
from pyspark.mllib.evaluation import RegressionMetrics

testdata = ratings.map(lambda p: (p.user, p.product))
predictions = model.predictAll(testdata).map(lambda r: ((r.user, r.product), r.rating))
ratesAndPreds = ratings.map(lambda r: ((r.user, r.product), r.rating)).join(predictions)
predictedAndTrue = ratesAndPreds.map(lambda r: r[1])
regressionMetrics = RegressionMetrics(predictedAndTrue)

# https://www.cnblogs.com/rener0424/p/11231107.html
print('explainedVariance is {:.5f}'.format(regressionMetrics.explainedVariance))
print('meanAbsoluteError is {:.5f}'.format(regressionMetrics.meanAbsoluteError))
print('meanSquaredError is {:.5f}'.format(regressionMetrics.meanSquaredError))
print('r2 is {:.5f}'.format(regressionMetrics.r2))
print('rootMeanSquaredError is {:.5f}'.format(regressionMetrics.rootMeanSquaredError))

# 5、为用户推荐电影

# 整理电影ID和名称数据，将推荐的电影ID翻译成电影名称
movies = sc.textFile("/Users/onlyone/python/ml100k/u.item")
movie_titles = movies.map(lambda line: line.split('|')[:2]).map(lambda line: (int(line[0]), line[1])).collectAsMap()

# 为用户 userId = 800寻找 K = 10个推荐点用
userId = 800
K = 8
# 方法一：为每个用户推荐K个电影，为每个电影推荐K个对它感兴趣的用户
# products_for_users = model.recommendProductsForUsers(K)  # 为每个用户推荐K个电影
# users_for_products = model.recommendUsersForProducts(K)  # 每个电影推荐K个对它感兴趣的用户
# products_for_users.lookup(userId)  # 查看给userId推荐的电影评分情况

# 方法二：模型自带的给用户user推荐Top K商品的使用方法
topKRecs = model.recommendProducts(user=userId, num=K)
print("-----------------------")
print('Top {} moives recommended for user {} are:'.format(K, userId))
for rec in topKRecs:
    print(movie_titles[rec.product], rec.rating)  # 查看给用户推荐的商品及得分

# 保存模型、可供下次调用
model.save(sc, "movie-recommend-model")
print(model)
